import re
import sys
import urllib2
import Queue

#===========GLOBALS=================#
max_depth = 2
i = 0
queues = []
link = "http://www.blank.com"
domain_name = ""
output_folder = "."
output_name = "ponies_"
output_file = ""
SCRIPT_NAME = 'pageGet.py'
#===========FUNCTIONS=================#

def save_all_url( link, next_queue ) :
#~~~~~~~~~~OPEN FILE AND ERROR TEST~~~~~~~~~~~~~#
    print ''
    print "-> CURRENT LINK [ " + link + " ] "
    domain_name = get_current_domain_name( link )
    page=""
    try:
        page=urllib2.urlopen( link )
    except IOError, e:
        if hasattr(e, 'reason'):
            print "Nous avons echoue  joindre le serveur."
            print "Raison: ", e.reason
        elif hasattr(e, 'code'):
            print "Le serveur n'a pu satisfaire la demande."
        print 'Code d\' erreur : ', e.code
        return

    data=page.read().split("</a>")
    tag="<a href=\""
    endtag="\""
    current_zelda = ""
#~~~~~~~~~~FOREACH LINK~~~~~~~~~~~~~#
    for item in data:
        if "<a href" in item:
            try:
                ind = item.index(tag)
                item=item[ind+len(tag):]
                end=item.index(endtag)
            except: pass
            else:
#~~~ WE NEED TO ADD THE CURRENT DOMAIN_NAME ??? ~~~#
                if  item[0] == '/' :
                     current_zelda = domain_name+item[:end]
                else :
                    if item[0] == 'h' and  item[1] == 't'  and item[2] == 't'  and item[3] == 'p' :
                        current_zelda = item[:end]
                    else:
                        current_zelda = domain_name+'/'+item[:end]

                print '~~~~~~~~'+current_zelda  
                if is_valid_extention( current_zelda ) :
                    write_output( current_zelda )
                    queues[next_queue].put( current_zelda )

    return

def is_valid_extention( link ):
    vaa = link.split('.')
    vaa = vaa[::-1]
    vaa = vaa[0]
    if vaa=='pdf' or vaa=='bin' or vaa=='mov' or vaa=='mp3' or vaa=='mp4' or vaa=='avi' or vaa=='divx' or vaa=='ogg' :
        return False
    else:
        return True

def write_output( link ):
    output_file.write(link+'\n')
    return

def get_current_domain_name( link ):
    rainbowdash = link.split('/')
    domain_name = rainbowdash[0]+'//'+rainbowdash[2]
    return domain_name

def print_help():
    print 'Usage: '+SCRIPT_NAME+" [--usage] [--help] [-h]"
    print "        [http://MYURL.TLD [MAX_DEPTH]]"
    return
#=============== MAIN =======================#

############## ARG ###############

argc = len( sys.argv )
if argc > 1:
    if sys.argv[1]=="--help" or sys.argv[1]=='-h' or sys.argv[1]=='--usage' :
        print_help()
        exit()
    domain_name = get_current_domain_name( sys.argv[1] )
    link = sys.argv[1]
if argc > 2:
    max_depth = int(sys.argv[2])



############### INIT LOOP ###################
for i in range(max_depth+1):
    queues.append(Queue.Queue())

queues[0].put( link )
i = 0

############### MAIN LOOP ###################
for i in range(max_depth):
    output_file = open( output_name+str(i),'w' )
    print ''
    print "|========> DEPTH : "+str(i)
    while not queues[i].empty():
        save_all_url( queues[i].get(), i+1 )
    output_file.close()
